{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import struct\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/Users/rench/code/data/MNIST/raw/train-images-idx3-ubyte\",\"rb\") as f:\n",
    "    file = f.read(24)\n",
    "\n",
    "#int.from_bytes(file[16:18], byteorder='big')\n",
    "ss = struct.unpack('>iiiiii',file)\n",
    "print(ss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST数据集\n",
    "第1-４个byte（字节，１byte=8bit），即前32bit存的是文件的magic number，对应的十进制大小是2051；\n",
    "第5-8个byte存的是number of images，即图像数量60000；\n",
    "第9-12个byte存的是每张图片行数/高度，即28；\n",
    "第13-16个byte存的是每张图片的列数/宽度，即28。\n",
    "从第17个byte开始，每个byte存储一张图片中的一个像素点的值。\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用struct读取二进制文件,注意大小端读写"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/Users/rench/code/data/MNIST/ubyte/train-images-idx3-ubyte\",\"rb\") as f:\n",
    "    file = f.read()\n",
    "    images = np.frombuffer(file, np.uint8, offset=16)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(linewidth=200)\n",
    "print(images.shape)\n",
    "data = images.reshape(-1, 28,28)\n",
    "print(data.shape)\n",
    "print(data[0])\n",
    "#data[0].reshape((28,28))[10:20,10:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets.MNIST import MNIST\n",
    "data_MNIST = MNIST()\n",
    "data_imgs, data_labs = data_MNIST.train_sets()\n",
    "print(data_imgs.shape)\n",
    "print(data_labs.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.3 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "72afde127a681a9c4291b943fce7674ba5dee3f782a7f671a875411cb5f75bf9"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
